#
# 
#

# Look for experiment data folders modified more recently than
# the timestamp of this file. For testing purposes, adjust as
#      touch -t 199501131000 /var/lib/dataMINX
# Additionally it contains a hashtable of experiment folder mod times
DATAMINX="/var/lib/dataMINX"

#mount -t cifs //192.168.97.140/csaf_others /mnt
FOLDERS=["/mnt/avon"]
FOLDERS=["/mnt/avon/mht11jp01"]
FOLDERS=["/mnt/Frames/csaf" ]
FOLDERS=["/mnt/Frames/csaf/act06pj14" ]
FOLDERS=["/mnt/Frames/Paul/afm09pj4"]
FOLDERS=["/mnt/Frames" ]

# folder for stashing generated ICAT XML files
ICATXMLDATA = "/home/icatdata"
# subfolder for failed XML ingest files
FAILED = "failed"

# recipients for screwed up email messages
EMAIL_ALERT_LIST=["someone@mail.host.net"]
EMAIL_SMTP_HOST="smtp.mail.host.net"
#        ssh dataminx -L 1521:localhost:1521
#ICATDBHOST = "localhost"    # via port-forward SSH tunnel
ICATDBHOST = "my.oracle.host.net"
ICATDBNAME = "XE"
ICATDBPORT = "1521"
ICATDBUSER = "icat"
ICATDBPASS = "pass"
#
ICATADMINURL = "https://icat.admin.ws.host.net/ICATAdminService/ICATAdmin"
ICATADMINUSER = "admin"
ICATADMINPASS = "pass"
#
ICATURL  = "https://icat.ws.host.net/ICATService/ICAT"
ICATUSER = "super"
ICATPASS = "pass"
#
PDBWSURL  = "http://proposalsdb.ws.host.net/?q=services/soap"
PDBWSUSER = "admin"
PDBWSPASS = "pass"
#

BRKDBNAME = 'BAXSdb'
BRKDBHOST = "bruker.postgresql.host.net"
BRKDBPORT = "5432"
BRKDBUSER = "BrukerPGSQL"
BRKDBPASS = "BrukerPGSQLpass"



import os
import os.path
import re
SMARTRE= re.compile('\.\d\d\d$')

# concerned about different versions of same file
# two separate uses of a single file extension 
# distinguishing between SMART and APEX and (Oxford) SUPERNOVA data 

# formats are all lower case for easy string comparison

class DataFormats:
  """Singleton class object? Not quite """
  # ---------------------------
  # changing these will lead to incompatible /var/lib/DATAMINX pickled py file
  APEX  = "fr591"
  SMART = "smart"
  NOVA  = "nova"
  # ---------------------------
  SUNDRIES ="SUNDRIES"
  INDEXING ="INDEXING"
  FILETYPES = {
     'raw' : {
        'regexp' : r'.*\.raw$',
        'version' : ['1'],
        'format' : 'bruker raw',
        'format_type' : 'ASCII',
        'description' : 'Bruker raw intensities harvested from one scan set',
     } ,
     'hkl' : {
        'regexp' : r'.*\.hkl$',  
        'version' : ['1'],  # HKLF3 /4 ???
        'format' : 'bruker hkl',
        'format_type' : 'ASCII',
        'description' : 'Interpolated reflection intensities from one or more scans ',
     } ,
     '_ls' : {
        'regexp' : r'.*\._ls$',
        'version' : ['1'],
        'format' : 'bruker _ls',
        'format_type' : 'ASCII',
        'description' : 'Bruker SAINT output listing ',
     } ,
     'sfrm' : {
        'regexp' : r'.*\.sfrm$',
        'version' : ['11','13','15'],
        'format' : 'bruker sfrm',
        'format_type' : 'binary',
        'handler' : ('sfrmProcess', 'SFRM'), 
        'description' : 'Bruker .SFRM image frame format',
     } ,
     'bg_snap' : {
        'regexp' : r'.*bg_snap_\d+_\d+\.sfrm$',
        'version' : ['11','13','15'],
        'format' : 'bruker sfrm',
        'format_type' : 'binary',
        'handler' : ('sfrmProcess', 'SFRM'), 
        'description' : 'Bruker .SFRM image background CCD frame',
     } ,
     'smart' : {
        'regexp' : r'.*\d\.\d\d\d$',
        'version' : ['9'],
        'format' : 'bruker smart',
        'handler' : ('sfrmProcess', 'SFRM'), 
        'format_type' : 'binary',
        'description' : 'Bruker SMART image frame format',
     } ,
     'bgsnap' : {
        'regexp' : r'.*bgsnap\d\d\.\d\d\d$',
        'version' : ['9'],
        'format' : 'bruker smart',
        'handler' : ('sfrmProcess', 'SFRM'), 
        'format_type' : 'binary',
        'description' : 'Bruker SMART image background frame format',
     } ,
     '_am' : {
        'regexp' : r'.*\._am$',
        'version' : ['9'],
        'format' : 'bruker _am',
        'handler' : ('sfrmProcess', 'SFRM'), 
        'format_type' : 'binary',
        'description' : 'Bruker Active Mask in .SFRM format',
     } ,
     '_ib' : {
        'regexp' : r'.*\._ib$',
        'version' : ['9'],
        'format' : 'bruker _ib',
        'handler' : ('sfrmProcess', 'SFRM'), 
        'format_type' : 'binary',
        'description' : 'Bruker Initial Background in .SFRM format',
     } ,
     '_dk' : {
        'regexp' : r'.*\._dk$',
        'version' : ['9'],
        'format' : 'bruker _dk',
        'handler' : ('sfrmProcess', 'SFRM'), 
        'format_type' : 'binary',
        'description' : 'Bruker DARK frame in .SFRM format',
     } ,
     '_fl' : {
        'regexp' : r'.*\._fl$',
        'version' : ['9'],
        'format' : 'bruker _fl',
        'handler' : ('sfrmProcess', 'SFRM'), 
        'format_type' : 'binary',
        'description' : 'Bruker Flood Table in .SFRM format',
     } ,
     'smart.ini' : {
        'regexp' : r'.*smart.ini$',
        'version' : ['1'],
        'format' : 'bruker smart.ini',
        'format_type' : 'ASCII',
        'description' : 'Bruker smart.ini config file',
     } ,
     '_pr' : {
        'regexp' : r'.*\._pr$',
        'version' : ['1'],
        'format' : 'bruker _pr',
        'format_type' : 'ASCII',
        'description' : 'Bruker pseudo .ins/.sfrm-header text file',
     } ,
     'abs' : {
        'regexp' : r'.*\.abs$',
        'version' : ['1'],
        'format' : 'bruker abs',
        'format_type' : 'ASCII',
        'description' : 'Bruker SADABS scaling and absorption correction log',
     } ,
     'prp' : {
        'regexp' : r'.*\.prp$',
        'version' : ['1'],
        'format' : 'bruker prp',
        'format_type' : 'ASCII',
        'description' : 'Bruker XPREP listing, text format',
     } ,
     'xht' : {
        'regexp' : r'.*\.xht$',
        'version' : ['1'],
        'format' : 'bruker xht',
        'format_type' : 'ASCII',
        'description' : 'Bruker scan analysis control file',
     } ,
     'cif' : {
        'regexp' : r'.*\.cif$',
        'version' : ['1.0'],
        'format' : 'cif',
        'format_type' : 'ASCII',  # CIF 2 will be UTF8
        'description' : 'Crystallographic Information File',
     } ,
     'pcf' : {
        'regexp' : r'.*\.pcf$',
        'version' : ['1.0'],
        'format' : 'bruker pcf',
        'format_type' : 'ASCII',
        'description' : 'Bruker Crystallographic Information File',
     } ,
     'res' : {
        'regexp' : r'.*\.res$',
        'version' : ['1'],
        'format' : 'shelx res',
        'format_type' : 'ASCII',
        'description' : 'SHELX result file',
     } ,
     'ins' : {
        'regexp' : r'.*\.ins$',
        'version' : ['1'],
        'format' : 'shelx ins',
        'format_type' : 'ASCII',
        'description' : 'SHELX ins file',
     } ,
     'p4p' : {
        'regexp' : r'.*\.p4p$',
        'version' : ['1'],
        'format' : 'bruker p4p',
        'format_type' : 'ASCII',
        'description' : 'Bruker SHELX ini file',
     } ,
     'eps' : {
        'regexp' : r'.*\.eps$',
        'version' : ['3.0'],
        'format' : 'epsf',
        'format_type' : 'ASCII',
        'description' : 'adobe postscript',
     } ,
     'png' : {
        'regexp' : r'.*\.png$',
        'version' : ['1'],
        'format' : 'png',
        'format_type' : 'binary',
        'description' : 'Portable network graphics',
     } ,
     'vzs' : {
        'regexp' : r'.*\.vzs$',
        'version' : ['1'],
        'format' : 'bruker vzs',
        'format_type' : 'binary',
        'description' : 'Bruker zipped mpeg?',
     } ,
     # -----------------------------------------------------------------
     'img' : {
        'regexp' : r'.*\.img$',
        'version' : ['3.0'],
        'format' : 'oxford  img',
        'handler' : ('oxfordProcess', 'OXFORD'), 
        'format_type' : 'binary',
        'description' : 'Oxford Diffraction .img CCD frame file',
     } ,
     'ini' : {
        'regexp' : r'.*\.ini$',
        'version' : ['1'],
        'format' : 'oxford ini',
        'format_type' : 'ASCII',
        'description' : 'Oxford Diffraction .ini configuration info',
     } ,
     'ini_report' : {
        'regexp' : r'.*\.ini_report$',
        'version' : ['1'],
        'format' : 'oxford ini_report',
        'format_type' : 'ASCII',
        'description' : 'Oxford Diffraction .ini configuration info',
     } ,
     'ccd' : {
        'regexp' : r'.*\.ccd$',
        'version' : ['1'],
        'format' : 'oxford  ccd',
        'format_type' : 'binary',
        'description' : 'Oxford Diffraction .ccd file',
     } ,
     'cif_od' : {
        'regexp' : r'.*\.cif_od$',
        'version' : ['1.0'],
        'format' : 'cif',
        'format_type' : 'ASCII', 
        'description' : 'Crystallographic Information File',
     } ,
     'par' : {
        'regexp' : r'.*\.par$',
        'version' : ['1.0'],
        'format' : 'oxford par',
        'format_type' : 'ASCII', 
        'description' : 'XCALIBUR PARAMETER FILE',
     } ,
     'jpg' : {
        'regexp' : r'.*\.jpg$',
        'version' : ['1'],
        'format' : 'jpeg',
        'format_type' : 'binary',
        'description' : 'Joint Photographic Experts Group image file'
     } ,
     'jpr' : {
        'regexp' : r'.*\.jpr$',
        'version' : ['1.000'],
        'format' : 'km4abs jpr',
        'format_type' : 'ASCII', 
        'description' : 'Oxford KM4ABS microscope movie settings',
     } ,
  }

  ARTIFACTS = {
    APEX : {
      'xml_template_new' : 'map-csaf_apex2icat.xml',
      'xml_template_upd' : 'map-csaf_apex2icat-upd.xml',
      'data_re' : re.compile('\.sfrm$') ,
      'data_field_sep' : '_', 
      'folders' : { 
            '.' : {
              'data' : [ ('sfrm' , ['11','13','15'] )  ] ,    
              'files' : [ 
                        ('vzs' , '' ),
                  ],
            },
            'work' : {
              'files' : [ 
                ('bg_snap' , ['11','13','15'], False ), # ignore it
                ('raw' , '1' ),
                ('hkl' , '1' ),
                ('_ls' , '1' ),
                ('_am' , '9' ),
                ('_ib' , '9' ),
                ('_dk' , '9' ),
                ('_fl' , '9' ),
                ('_pr' , '1' ),
                ('abs' , '1' ),
                ('prp' , '1' ),
                ('xht' , '1' ),
                ('cif' , '1.0' ),
                ('pcf' , '1.0' ),
                ('ins' , '1' ),
                ('res' , '1' ),
                ('p4p' , '1' ),
                ('eps' , '3.0' ),
                ('png' , '1' ),
                ('vzs' , '1' ),
              ], 
            },  # end of work subfolder 
            'struct' : {    # subfolder
              'files' : [ ('cif' , '1.0'), ('res' , '1'), ('hkl' , '1') ],
        },  
      },  # end of folder list
    },

    SMART : {
      'xml_template_new' : 'map-csaf_apex2icat.xml',
      'xml_template_upd' : 'map-csaf_apex2icat-upd.xml',
      'data_re' : re.compile('\.\d\d\d$'),
      'data_field_sep' : '.', 
      'folders' : { 
        '.' : {
          'data' : [ ('smart' , '9') ] ,    
          'files' : [ 
                ('vzs' , '' ),
                ('p4p' , '1' ),
                ('smart.ini' , '1' ),
           ],
        },
        'work' : {
              'files' : [ 
                ('raw' , '1' ),
                ('bgsnap', '9', False), # ignore it
                ('hkl' , '1' ),
                ('_ls' , '1' ),
                ('_am' , '9' ),
                ('_ib' , '9' ),
                ('_dk' , '9' ),
                ('_fl' , '9' ),
                ('_pr' , '1' ),
                ('abs' , '1' ),
                ('prp' , '1' ),
                ('xht' , '1' ),
                ('cif' , '1.0' ),
                ('pcf' , '1.0' ),
                ('ins' , '1' ),
                ('res' , '1' ),
                ('p4p' , '1' ),
                ('eps' , '3.0' ),
                ('png' , '1' ),
                ('vzs' , '1' ),
              ], 
        },  # end of work  
        'struct' : {       # subfolder
              'files' : [ ('cif' , '1.0'), ('res' , '1'), ('hkl' , '1') ],
        },  
      },  # end of folder list
    },
    NOVA : {
      'xml_template_new' : 'map-csaf_nova2icat.xml',
      'xml_template_upd' : 'map-csaf_nova2icat.xml',
      'data_re' : re.compile('\.img$') ,
      'data_field_sep' : '_', 
      'folders' : { 
        '.' : {
          'files' : [ 
                ('ccd' , '' ),
#                ('red' , '' ),
                ('cif' , '' ),
                ('cif_od' , '' ),
                ('hkl' , '' ),
                ('ins' , '' ),
                ('p4p' , '' ),
                ('par' , '' ),
          ],

        },
        'frames' : {
              'data' : [ ('img' , '3') ] ,    
        }, 
        'movie' : {
              'files' : [ 
                ('jpg' , '' ),
                ('jpr' , '' ),
              ],
        }, 
        'expinfo' : {
              'files' : [ 
                ('ini' , '' ),
                ('ini_report' , '' ),
              ], 
        }, 
        'log' : {
              'files' : [ 
              ], 
        }, 
        'plots_dc' : {
              'files' : [ 
              ], 
        }, 
        'plots_red' : {
              'files' : [ 
              ], 
        }, 
        'struct/tmp' : {
              'files' : [ 
                ('cif' , '' ),
              ], 
        }, 
      },  # end of folder list
    }, # end of NOVA
  }

  def someClassFunc():
    """ do somehting """
    pass


bruker_scan_types = {
     1:  "None",
     2:  "Omega scan",
     3:  "Phi scan",
     9:  "Still scan",
     12: "Phi360 scan",
     14: "Theta/2Theta scan",
     15: "Chi scan" 
}

basic_dataset = {
     'proto':   None, 
     'num':    None,      # len(filelist)
     'mach':   None,      # type
     'dsetname': None,    # dsetname,
     'files' : [],
     'params': [] ,
     'scan':  None,  # scantype 
}


"""
dset['_diffrn_scan.frame_id_start'] = files[0]['name']
dset['_diffrn_scan.frame_id_end']  = files[-1]['name']
"""


bruker_scan_params = [
          {'name': '_diffrn_measurement.method',
                        'sval': None,     #types[op[0]],
                        'units': 'n/a'},
          {'name': '_diffrn_measurement.sample_detector_distance',
                        'nval': None,     #op[2],
                        'units': 'mm'},
          {'name': '_diffrn_scan.integration_time',
                        'nval': None,     #op[1],
                        'units': 's'},
          {'name': '_diffrn_scan.SPEED',
                        'nval': None,     #speed,
                        'units': u'\xb0/s'},
          {'name': '_diffrn_scan.id',
                        'sval': None,     #n+1,
                        'units': 'n/a'},
          {'name': '_diffrn_scan.AXIS_ANGLE_RANGE',
                        'nval': None,     #op[3],
                        'units': u'\xb0'},
          {'name': '_diffrn_scan.AXIS_ANGLE_DIRECTION',
                        'nval': None,     #op[4],
                        'units': 'n/a'},
          {'name': '_diffrn_scan.AXIS_ANGLE_INCREMENT',
                        'nval': None,     #op[5],
                        'units': u'\xb0'},
          {'name': '_diffrn_scan.DETECTOR_AXIS_ANGLE_TWOTHETA',
                        'nval': None,     #op[6],
                        'units': u'\xb0'},
          {'name': '_diffrn_scan.GONIOMETER_AXIS_ANGLE_OMEGA',
                        'nval': None,     #op[7],
                        'units': u'\xb0'},
          {'name': '_diffrn_scan.GONIOMETER_AXIS_ANGLE_CHI',
                        'nval': None,     #op[8],
                        'units': u'\xb0'},
          {'name': '_diffrn_scan.GONIOMETER_AXIS_ANGLE_PHI',
                        'nval': None,     #op[9],
                        'units': u'\xb0'},
          {'name': '_diffrn_scan.frames',
                     'nval': None,     #len(filelist),
                     'units': 'n/a'}, 
        ]




basic_file    = {
     'dir':    None,      # file or string?
     'name':    None,      # file or string?
     'params':  [ ],
     'elements': [ ] 
}


bruker_data_file = {
    'dir':   None,
    'name':  None, 
    'params': [ 
        {'name': '_diffrn_scan_frame.frame_id',
                        'sval': None,   # filename,
                        'units': 'n/a'},
        {'name': '_diffrn_scan_frame.scan_id',
                        'nval': None,    # sid,
                        'units': 'n/a'},
        {'name': '_diffrn_scan_frame.frame_number',
                        'nval': None,    # i,
                        'units': 'n/a'},
    ]
}


class Artifacts(DataFormats):
  """Singleton class object """
 
  @staticmethod
  def getInstruments():
    """ return list of instrument keys """
    if not DataFormats.ARTIFACTS:
      return None
    insts = DataFormats.ARTIFACTS.keys()
    return insts
   
  @staticmethod
  def getInstrumentFolders(instrument):
    if not DataFormats.ARTIFACTS.has_key(instrument): 
      return None
    inst = DataFormats.ARTIFACTS[instrument]
    return inst['folders'].keys()


  @staticmethod
  def getInstrumentDataFolder(instrument):
    """Primary folder holding bulk of raw data"""
    if not DataFormats.ARTIFACTS.has_key(instrument): 
      return None
    inst = DataFormats.ARTIFACTS[instrument]
    if inst.has_key('data'):
       return (".", inst)    # top directory
    folders =  inst['folders']
    for folder in folders.keys():
      if folders[folder].has_key('data'):
         return (folder, folders[folder]) 
    return None

  @staticmethod
  def getREforFileType(ftype):
    filetypes = DataFormats.FILETYPES 

    if not filetypes.has_key(ftype):
      # the config edit has gone wrong somewhere!
      raise Exception, "DataFormats.FILETYPES has no type: " + str(ftype)
    typeinfo = filetypes[ftype] 
    regexp = typeinfo['regexp']
    #print regexp
    #regex = re.compile(regexp)
    return regexp


  @staticmethod
  def getFolderArtifacts(instrument, folder):
    if not DataFormats.ARTIFACTS.has_key(instrument): 
      return None
    inst = DataFormats.ARTIFACTS[instrument]
    if not folder:
      return inst  # just return collection root
    folders =  inst['folders']
    if not folders.has_key(folder): 
      return None
    return folders[folder]
    

  @staticmethod
  def getFolderFileREs(instrument, folder):
    artifacts = Artifacts.getFolderArtifacts(instrument,folder)

    re_list = []
    filetypes = DataFormats.FILETYPES 

    if artifacts.has_key('data'):
      data = artifacts['data'] 
      for type, version in data:
        if not filetypes.has_key(type):
          # the config edit has gone wrong somewhere!
          raise Exception, "DataFormat.FILETYPES has no type: " + type
        typeinfo = filetypes[type] 
        # or the whole typeinfo record????
        re_list.append( (type, typeinfo['regexp'] ) )

    if artifacts.has_key('files'):
      files = artifacts['files'] 
      for info in files:   #  ( type, version, [exclude?] )
        type = info[0]
        if not filetypes.has_key(type):
          # the config edit has gone wrong somewhere!
          raise Exception, "DataFormat.FILETYPES has no type: " + type
        typeinfo = filetypes[type] 
        # or the whole typeinfo record????
        re_list.append( (type, typeinfo['regexp'] ) )
    
    return re_list

# compile all the regular expressions for efficiency
for key in DataFormats.FILETYPES.keys():
  regexp = DataFormats.FILETYPES[key]['regexp']
  DataFormats.FILETYPES[key]['regexp'] = re.compile(regexp)
  
